/**
 * @description JSON tokenizer
 *
 *
 */
Jx.categorize('code.json.tokenizer', {

	$extend: 'Jx.lib.Tokenizer',

	states: {

		'start': {
			'"': 'str_start',
			'\'': 'str_start',
			'[': 'open_brace',
			']': 'close_brace',
			'{': 'open_brace',
			'}': 'close_brace',
			'(': 'open_brace',
			')': 'close_brace',
			':': 'colon',
			'[*]': 'ident_start',
			'0': 'number',
			'1': 'number',
			'2': 'number',
			'3': 'number',
			'4': 'number',
			'5': 'number',
			'6': 'number',
			'7': 'number',
			'8': 'number',
			'9': 'number',
			'-': 'negative',
			'+': 'positive',
			'.': 'dot',
			',': 'comma',
			'\r': 'whitespace',
			'\n': 'whitespace',
			'\t': 'whitespace',
			' ': 'whitespace',
			'/': 'start_comment'
		},

		'start_comment': {
			'*': 'multiline_comment',
			'/': 'singleline_comment'
		},

		'multiline_comment': {
			'[*]': 'multiline_comment',
			'*': 'ending_multiline_comment'
		},
		'ending_multiline_comment': {
			'*': 'ending_multiline_comment',
			'/': 'end_multiline_comment',
			'[*]': 'multiline_comment'
		},

		'singleline_comment': {
			'\n': 'end_singleline_comment',
			'[*]': 'singleline_comment'
		},


		'str_start': {
			'\\': 'esc_start',
			'[*]': 'str_values',
			'"': 'str_end',
			'\'': 'str_end'
		},

		'str_values': {

			'\\': 'esc_start',
			'[*]': 'str_values',
			'"': 'str_end',
			'\'': 'str_end'
		},

		'esc_start': {
			'0': 'decide_esc_digit',
			'1': 'decide_esc_digit',
			'2': 'decide_esc_digit',
			'3': 'decide_esc_digit',
			'4': 'decide_esc_digit',
			'5': 'decide_esc_digit',
			'6': 'decide_esc_digit',
			'7': 'decide_esc_digit',
			'8': 'decide_esc_digit',
			'9': 'decide_esc_digit',
			'x': 'esc_hex',
			'u': 'esc_unicode',
			'[*]': 'esc_etc'

		},

		'esc_hex': {},

		'esc_unicode': {},

		'decide_hex_value': {
			'[*]': 'decide_hex_value'
		},

		'decide_utf8_value': {
			'[*]': 'decide_utf8_value'
		},

		'decide_esc_digit': {
			'[*]': 'decide_esc_digit'
		},

		'ident_start': {
			'[*]': 'ident_match'
		},

		'ident_match': {
			'[*]': 'ident_match'
		},

		'str_end': {},

		'dot': {
			'0': 'fraction',
			'1': 'fraction',
			'2': 'fraction',
			'3': 'fraction',
			'4': 'fraction',
			'5': 'fraction',
			'6': 'fraction',
			'7': 'fraction',
			'8': 'fraction',
			'9': 'fraction'
		},

		'number': {
			'0': 'number',
			'1': 'number',
			'2': 'number',
			'3': 'number',
			'4': 'number',
			'5': 'number',
			'6': 'number',
			'7': 'number',
			'8': 'number',
			'9': 'number',
			'.': 'fraction_start',
			'e': 'exp',
			'E': 'exp'
		},

		'fraction_start': {
			'0': 'fraction',
			'1': 'fraction',
			'2': 'fraction',
			'3': 'fraction',
			'4': 'fraction',
			'5': 'fraction',
			'6': 'fraction',
			'7': 'fraction',
			'8': 'fraction',
			'9': 'fraction'
		},

		'fraction': {
			'0': 'fraction',
			'1': 'fraction',
			'2': 'fraction',
			'3': 'fraction',
			'4': 'fraction',
			'5': 'fraction',
			'6': 'fraction',
			'7': 'fraction',
			'8': 'fraction',
			'9': 'fraction',
			'e': 'exp',
			'E': 'exp'
		},

		'exp': {
			'-': 'exp_start_number',
			'+': 'exp_start_number',
			'0': 'exp_number',
			'1': 'exp_number',
			'2': 'exp_number',
			'3': 'exp_number',
			'4': 'exp_number',
			'5': 'exp_number',
			'6': 'exp_number',
			'7': 'exp_number',
			'8': 'exp_number',
			'9': 'exp_number'
		},

		'exp_start_number': {

			'0': 'exp_number',
			'1': 'exp_number',
			'2': 'exp_number',
			'3': 'exp_number',
			'4': 'exp_number',
			'5': 'exp_number',
			'6': 'exp_number',
			'7': 'exp_number',
			'8': 'exp_number',
			'9': 'exp_number'

		},

		'exp_number': {

			'0': 'exp_number',
			'1': 'exp_number',
			'2': 'exp_number',
			'3': 'exp_number',
			'4': 'exp_number',
			'5': 'exp_number',
			'6': 'exp_number',
			'7': 'exp_number',
			'8': 'exp_number',
			'9': 'exp_number'

		},

		'whitespace': {
			'\r': 'whitespace',
			'\n': 'whitespace',
			'\t': 'whitespace',
			' ': 'whitespace'
		},

		'negative': {
			'0': 'number',
			'1': 'number',
			'2': 'number',
			'3': 'number',
			'4': 'number',
			'5': 'number',
			'6': 'number',
			'7': 'number',
			'8': 'number',
			'9': 'number'
		},
		'positive': {
			'0': 'number',
			'1': 'number',
			'2': 'number',
			'3': 'number',
			'4': 'number',
			'5': 'number',
			'6': 'number',
			'7': 'number',
			'8': 'number',
			'9': 'number'
		},

		'open_brace': {},
		'close_brace': {},
		'colon': {},

		'end_ident': {},
		'end_singleline_comment': {},
		'end_multiline_comment': {},
		'comma': {}


	},

	stateCallback: {

		'str_start': function(chr, input, state, previousState){

			//-- dont include " character
			this.strQuote = chr;

		},


		'esc_start': function(chr, input, state, previousState){

			//-- dont include '\' character

		},

		'esc_hex': function(chr, input, state, previousState){

			//-- dont include '\x' character
			this.state = 'decide_hex_value';

			this.escaped = '';

		},

		'esc_unicode': function(chr, input, state, previousState){

			//-- dont include '\u' character
			this.state = 'decide_utf8_value';

			this.escaped = '';

		},

		'decide_hex_value': function(chr, input, state, previousState){

			this.endEscapeCodeIf('hex', 2, 16, chr);

		},
		'decide_utf8_value': function(chr, input, state, previousState){

			this.endEscapeCodeIf('hex', 4, 16, chr);

		},
		'decide_esc_digit': function(chr, input, state, previousState){

			this.endEscapeCodeIf('octal', 3, 8, chr);

		},

		'str_end': function(chr, input, state, previousState){

			var ret;

			if (chr == !this.strQuote){

				this.appendBuffer(chr);

				this.state = previousState;

			} else {

				//-- check if it matches "date"
				ret = this.dateRe.test(this.getCurrentBuffer()) ?

					'date'

					:

					'string';

			}

			return ret;

		},

		'esc_etc': function(chr, input, state, previousState){

			var esc = this.escapeChars;

			this.appendBuffer(

				chr in esc ?

					esc[chr]

					:

					chr

			);

			this.state = 'str_values';

		},

		'open_brace': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return chr;

		},

		'close_brace': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return chr;

		},

		'colon': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return chr;

		},

		'ident_start': function(chr, input, state, previousState){

			return this.identMatchCallback(chr, input, state, previousState);

		},



		'ident_match': function(chr, input, state, previousState){

			return this.identMatchCallback(chr, input, state, previousState);

		},

		'number': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return 'number';

		},

		'fraction': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return 'number';

		},

		'exp_number': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return 'number';

		},

		'comma': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return chr;

		},

		'whitespace': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return 'whitespace';

		},

		'end_singleline_comment': function(chr, input, state, previousState){

			if (chr != '\r'){

				this.appendBuffer(chr);

			}

			return 'singleline_comment';

		},

		'end_multiline_comment': function(chr, input, state, previousState){

			return 'multiline_comment';

		}


	},

	identStartRe: /[\$A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]/,

	identSuffixRe: /[\$A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-0-9\u00B7\u0300-\u036F\u203F-\u2040]/,

	dateRe: /^([0-9]{4})\-([0-9]{2})\-([0-9]{2})T([0-9]{2})\:([0-9]{2})\:([0-9]{2})Z$/,

	hexRe: /^[0-9a-f]+$/i,

	octalRe: /^[0-9]+$/i,

	escaped: '',

	escapeChars: {
		'b': '\b',
		'f': '\f',
		'n': '\n',
		'O': '\O',
		'r': '\r',
		't': '\t',
		'v': '\v'
	},

	literals: {

		'null': true,

		'object': true,

		'true': true,

		'false': true,

		'function': true,

		'undefined': true

	},

	identFound: undefined,

	strQuote: undefined,

	constructor: function(){

		this.identFound = '';

		this.strQuote = '';

		this.$super(arguments);

	},

	onAfterIterate: function(){

		this.identFound = '';

		this.strQuote = '';

		this.$super();

	},

	endEscapeCodeIf: function(re, limit, base, chr){

		var notValid = !this[re+'Re'].test(chr),

			escaped;

		if (!notValid){

			this.escaped+=chr;

		}

		escaped = this.escaped;

		//-- exit
		if (escaped.length == limit || notValid){

			if (notValid){

				this.backtrack(1);

			}

			this.appendBuffer(

				String.fromCharCode(parseInt(escaped, base))

			);

			this.escaped = '';

			this.state = 'str_values';

		}

	},

	identMatchCallback: function(chr, input, state, previousState){

		var ret = void(0),

			ident;

		//-- proceed
		if (this.identSuffixRe.test(chr)){

			this.appendBuffer(chr);

			this.state = 'ident_match';

			ident = this.identFound+=chr;

			ret = this.literals.hasOwnProperty(ident) ?

				ident

				:

				this.dateRe.test(ident) ?

					'date'

					:

					'ident';

		} else {

			this.state = 'end_ident';


		}

		return ret;

	},

	reset: function(){

		delete this.escaped;

		return this.$super(arguments);

	}

});
